{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sklearn\n",
    "import numpy as np\n",
    "from scipy import optimize\n",
    "from scipy.optimize import brute\n",
    "from scipy.optimize import minimize\n",
    "from scipy.optimize import minimize_scalar\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tteofili/.local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3063: DtypeWarning: Columns (9) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LoanNr_ChkDgt</th>\n",
       "      <th>Name</th>\n",
       "      <th>City</th>\n",
       "      <th>State</th>\n",
       "      <th>Zip</th>\n",
       "      <th>Bank</th>\n",
       "      <th>BankState</th>\n",
       "      <th>NAICS</th>\n",
       "      <th>ApprovalDate</th>\n",
       "      <th>ApprovalFY</th>\n",
       "      <th>...</th>\n",
       "      <th>RevLineCr</th>\n",
       "      <th>LowDoc</th>\n",
       "      <th>ChgOffDate</th>\n",
       "      <th>DisbursementDate</th>\n",
       "      <th>DisbursementGross</th>\n",
       "      <th>BalanceGross</th>\n",
       "      <th>MIS_Status</th>\n",
       "      <th>ChgOffPrinGr</th>\n",
       "      <th>GrAppv</th>\n",
       "      <th>SBA_Appv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000014003</td>\n",
       "      <td>ABC HOBBYCRAFT</td>\n",
       "      <td>EVANSVILLE</td>\n",
       "      <td>IN</td>\n",
       "      <td>47711</td>\n",
       "      <td>FIFTH THIRD BANK</td>\n",
       "      <td>OH</td>\n",
       "      <td>451120</td>\n",
       "      <td>28-Feb-97</td>\n",
       "      <td>1997</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28-Feb-99</td>\n",
       "      <td>$60,000.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>P I F</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>$60,000.00</td>\n",
       "      <td>$48,000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000024006</td>\n",
       "      <td>LANDMARK BAR &amp; GRILLE (THE)</td>\n",
       "      <td>NEW PARIS</td>\n",
       "      <td>IN</td>\n",
       "      <td>46526</td>\n",
       "      <td>1ST SOURCE BANK</td>\n",
       "      <td>IN</td>\n",
       "      <td>722410</td>\n",
       "      <td>28-Feb-97</td>\n",
       "      <td>1997</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31-May-97</td>\n",
       "      <td>$40,000.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>P I F</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>$40,000.00</td>\n",
       "      <td>$32,000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000034009</td>\n",
       "      <td>WHITLOCK DDS, TODD M.</td>\n",
       "      <td>BLOOMINGTON</td>\n",
       "      <td>IN</td>\n",
       "      <td>47401</td>\n",
       "      <td>GRANT COUNTY STATE BANK</td>\n",
       "      <td>IN</td>\n",
       "      <td>621210</td>\n",
       "      <td>28-Feb-97</td>\n",
       "      <td>1997</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31-Dec-97</td>\n",
       "      <td>$287,000.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>P I F</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>$287,000.00</td>\n",
       "      <td>$215,250.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000044001</td>\n",
       "      <td>BIG BUCKS PAWN &amp; JEWELRY, LLC</td>\n",
       "      <td>BROKEN ARROW</td>\n",
       "      <td>OK</td>\n",
       "      <td>74012</td>\n",
       "      <td>1ST NATL BK &amp; TR CO OF BROKEN</td>\n",
       "      <td>OK</td>\n",
       "      <td>0</td>\n",
       "      <td>28-Feb-97</td>\n",
       "      <td>1997</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30-Jun-97</td>\n",
       "      <td>$35,000.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>P I F</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>$35,000.00</td>\n",
       "      <td>$28,000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000054004</td>\n",
       "      <td>ANASTASIA CONFECTIONS, INC.</td>\n",
       "      <td>ORLANDO</td>\n",
       "      <td>FL</td>\n",
       "      <td>32801</td>\n",
       "      <td>FLORIDA BUS. DEVEL CORP</td>\n",
       "      <td>FL</td>\n",
       "      <td>0</td>\n",
       "      <td>28-Feb-97</td>\n",
       "      <td>1997</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14-May-97</td>\n",
       "      <td>$229,000.00</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>P I F</td>\n",
       "      <td>$0.00</td>\n",
       "      <td>$229,000.00</td>\n",
       "      <td>$229,000.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   LoanNr_ChkDgt                           Name          City State    Zip  \\\n",
       "0     1000014003                 ABC HOBBYCRAFT    EVANSVILLE    IN  47711   \n",
       "1     1000024006    LANDMARK BAR & GRILLE (THE)     NEW PARIS    IN  46526   \n",
       "2     1000034009          WHITLOCK DDS, TODD M.   BLOOMINGTON    IN  47401   \n",
       "3     1000044001  BIG BUCKS PAWN & JEWELRY, LLC  BROKEN ARROW    OK  74012   \n",
       "4     1000054004    ANASTASIA CONFECTIONS, INC.       ORLANDO    FL  32801   \n",
       "\n",
       "                            Bank BankState   NAICS ApprovalDate ApprovalFY  \\\n",
       "0               FIFTH THIRD BANK        OH  451120    28-Feb-97       1997   \n",
       "1                1ST SOURCE BANK        IN  722410    28-Feb-97       1997   \n",
       "2        GRANT COUNTY STATE BANK        IN  621210    28-Feb-97       1997   \n",
       "3  1ST NATL BK & TR CO OF BROKEN        OK       0    28-Feb-97       1997   \n",
       "4        FLORIDA BUS. DEVEL CORP        FL       0    28-Feb-97       1997   \n",
       "\n",
       "   ...  RevLineCr  LowDoc  ChgOffDate  DisbursementDate  DisbursementGross  \\\n",
       "0  ...          N       Y         NaN         28-Feb-99        $60,000.00    \n",
       "1  ...          N       Y         NaN         31-May-97        $40,000.00    \n",
       "2  ...          N       N         NaN         31-Dec-97       $287,000.00    \n",
       "3  ...          N       Y         NaN         30-Jun-97        $35,000.00    \n",
       "4  ...          N       N         NaN         14-May-97       $229,000.00    \n",
       "\n",
       "   BalanceGross  MIS_Status ChgOffPrinGr        GrAppv      SBA_Appv  \n",
       "0        $0.00        P I F       $0.00    $60,000.00    $48,000.00   \n",
       "1        $0.00        P I F       $0.00    $40,000.00    $32,000.00   \n",
       "2        $0.00        P I F       $0.00   $287,000.00   $215,250.00   \n",
       "3        $0.00        P I F       $0.00    $35,000.00    $28,000.00   \n",
       "4        $0.00        P I F       $0.00   $229,000.00   $229,000.00   \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('SBAnational.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_columns = ['LoanNr_ChkDgt', 'NAICS', 'ApprovalDate', 'DisbursementDate', 'Name', 'FranchiseCode']\n",
    "target = 'MIS_Status'\n",
    "categorical = ['City', 'State', 'Zip', 'Bank', 'BankState', 'NewExist', 'UrbanRural', 'RevLineCr', 'LowDoc']\n",
    "ordinal = ['ApprovalFY', 'Term', 'NoEmp', 'CreateJob', 'RetainedJob']\n",
    "money_columns = ['DisbursementGross', 'BalanceGross', 'ChgOffPrinGr', 'GrAppv', 'SBA_Appv']\n",
    "date_col = ['ChgOffDate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop bad columns\n",
    "df = df.dropna()\n",
    "df = df.drop(columns=bad_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert date to ordinal days\n",
    "import datetime as dt\n",
    "for c in date_col:\n",
    "    df[c] = pd.to_datetime(df[c]).map(dt.datetime.toordinal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['CHGOFF' 'P I F']\n"
     ]
    }
   ],
   "source": [
    "# encode labels\n",
    "le = preprocessing.LabelEncoder()\n",
    "df[target] = le.fit_transform(df[target].astype(str))\n",
    "class_names = le.classes_\n",
    "print(class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process ordinal features\n",
    "for col in ordinal:\n",
    "    df[col] = pd.to_numeric(df['ApprovalFY'].replace('1976A','1976', regex=False).replace('\\d+\\-\\w+\\-\\d+|,','', regex=True))\n",
    "    df[col] = df[col].astype('int32')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process money columns\n",
    "for c in money_columns:\n",
    "    df[c] = df[c].replace('\\$|,','', regex=True).replace('\\(','-', regex=True).replace('\\)','', regex=True)\n",
    "    df[c] = pd.to_numeric(df[c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 8, 11, 12, 13]\n"
     ]
    }
   ],
   "source": [
    "# process categorical features\n",
    "cat_idxs = [df.columns.get_loc(c) for c in categorical if c in df]\n",
    "print(cat_idxs)\n",
    "categorical_names = {}\n",
    "for c in categorical:\n",
    "    le = preprocessing.LabelEncoder()\n",
    "    df[c] = le.fit_transform(df[c])\n",
    "    categorical_names[c] = le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = df[target]\n",
    "X = df.drop(columns=[target])\n",
    "\n",
    "# Split the data into train and test data:\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[31169    41]\n",
      " [  944    21]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98     31210\n",
      "           1       0.34      0.02      0.04       965\n",
      "\n",
      "    accuracy                           0.97     32175\n",
      "   macro avg       0.65      0.51      0.51     32175\n",
      "weighted avg       0.95      0.97      0.96     32175\n",
      "\n",
      "0.9693861693861694\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(max_depth=19, random_state=0, n_estimators=10)\n",
    "rf.fit(X_train, Y_train)\n",
    "sklearn.metrics.accuracy_score(Y_test, rf.predict(X_test))\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "out = rf.predict(X_test) \n",
    "\n",
    "print(confusion_matrix(Y_test,out))\n",
    "print(classification_report(Y_test,out))\n",
    "print(accuracy_score(Y_test, out))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Counterfactual loss function = $L(x,x^\\prime,y^\\prime,\\lambda)=\\lambda\\cdot(\\hat{f}(x^\\prime)-y^\\prime)^2+d(x,x^\\prime)$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_f=X_train.shape[1]\n",
    "x = X_train.values[0].reshape(-1,20)\n",
    "y = rf.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial import distance\n",
    "def d(a,b):\n",
    "    return distance.cityblock(a, b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample a cf instance from the data distribution delimited by the (min, max) per feature boundaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cf_loss(xp, l, x, y):\n",
    "    xp = xp.reshape(-1,20)\n",
    "    y_c = rf.predict(xp)\n",
    "    diff = y_c-abs(1-y)\n",
    "    distance = d(xp, x)\n",
    "    loss = l * (diff)**2 + distance\n",
    "    return loss[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using lambda=0.0001\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11236.317031\n",
      "         Iterations: 1696\n",
      "         Function evaluations: 2439\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 169.987586\n",
      "         Iterations: 3008\n",
      "         Function evaluations: 3995\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 26.278540\n",
      "         Iterations: 1736\n",
      "         Function evaluations: 2440\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 3.261424\n",
      "         Iterations: 2788\n",
      "         Function evaluations: 3712\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 2.295628\n",
      "         Iterations: 1235\n",
      "         Function evaluations: 1796\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.395265\n",
      "         Iterations: 1936\n",
      "         Function evaluations: 2678\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.090509\n",
      "         Iterations: 1213\n",
      "         Function evaluations: 1743\n",
      "using lambda=0.001\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 13812.088900\n",
      "         Iterations: 1066\n",
      "         Function evaluations: 1652\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 184.668512\n",
      "         Iterations: 2689\n",
      "         Function evaluations: 3611\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 38.562037\n",
      "         Iterations: 2765\n",
      "         Function evaluations: 3771\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.390547\n",
      "         Iterations: 1676\n",
      "         Function evaluations: 2327\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 6.909562\n",
      "         Iterations: 2067\n",
      "         Function evaluations: 2829\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 2.824540\n",
      "         Iterations: 2149\n",
      "         Function evaluations: 2879\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1.260407\n",
      "         Iterations: 2735\n",
      "         Function evaluations: 3715\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1.084521\n",
      "         Iterations: 1483\n",
      "         Function evaluations: 2073\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1.071347\n",
      "         Iterations: 909\n",
      "         Function evaluations: 1382\n",
      "using lambda=0.01\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 41.256572\n",
      "         Iterations: 1428\n",
      "         Function evaluations: 2036\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 15.241741\n",
      "         Iterations: 2453\n",
      "         Function evaluations: 3325\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10.781103\n",
      "         Iterations: 2464\n",
      "         Function evaluations: 3329\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 2.006735\n",
      "         Iterations: 2365\n",
      "         Function evaluations: 3207\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.135665\n",
      "         Iterations: 1980\n",
      "         Function evaluations: 2678\n",
      "using lambda=0.1\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44.251051\n",
      "         Iterations: 1929\n",
      "         Function evaluations: 2708\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32.752897\n",
      "         Iterations: 2985\n",
      "         Function evaluations: 3976\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 26.493841\n",
      "         Iterations: 2241\n",
      "         Function evaluations: 3038\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 5.415496\n",
      "         Iterations: 1336\n",
      "         Function evaluations: 1952\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 3.567501\n",
      "         Iterations: 1696\n",
      "         Function evaluations: 2370\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 3.360828\n",
      "         Iterations: 944\n",
      "         Function evaluations: 1417\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 2.324103\n",
      "         Iterations: 1798\n",
      "         Function evaluations: 2498\n",
      "using lambda=1.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 17972.828983\n",
      "         Iterations: 1351\n",
      "         Function evaluations: 2019\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 276.652414\n",
      "         Iterations: 2024\n",
      "         Function evaluations: 2835\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 73.288589\n",
      "         Iterations: 1780\n",
      "         Function evaluations: 2474\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 22.773219\n",
      "         Iterations: 1356\n",
      "         Function evaluations: 1951\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 5.446038\n",
      "         Iterations: 2222\n",
      "         Function evaluations: 3022\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 4.816969\n",
      "         Iterations: 1732\n",
      "         Function evaluations: 2402\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 2.968053\n",
      "         Iterations: 1585\n",
      "         Function evaluations: 2192\n",
      "using lambda=10.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 6329.007701\n",
      "         Iterations: 1263\n",
      "         Function evaluations: 1928\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 14.506515\n",
      "         Iterations: 1905\n",
      "         Function evaluations: 2619\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.782391\n",
      "         Iterations: 1349\n",
      "         Function evaluations: 1913\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.542835\n",
      "         Iterations: 1062\n",
      "         Function evaluations: 1546\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.625019\n",
      "         Iterations: 885\n",
      "         Function evaluations: 1350\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.714509\n",
      "         Iterations: 826\n",
      "         Function evaluations: 1270\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.811143\n",
      "         Iterations: 896\n",
      "         Function evaluations: 1353\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 11.910025\n",
      "         Iterations: 688\n",
      "         Function evaluations: 1105\n",
      "using lambda=100.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 6131.558364\n",
      "         Iterations: 2199\n",
      "         Function evaluations: 3049\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 110.471429\n",
      "         Iterations: 2155\n",
      "         Function evaluations: 2955\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 108.388550\n",
      "         Iterations: 2354\n",
      "         Function evaluations: 3156\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 108.953219\n",
      "         Iterations: 1988\n",
      "         Function evaluations: 2698\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 109.471472\n",
      "         Iterations: 2270\n",
      "         Function evaluations: 3075\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 110.035410\n",
      "         Iterations: 1287\n",
      "         Function evaluations: 1835\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 111.029302\n",
      "         Iterations: 750\n",
      "         Function evaluations: 1168\n",
      "using lambda=1000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 9926.033140\n",
      "         Iterations: 2169\n",
      "         Function evaluations: 3089\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 4999.770526\n",
      "         Iterations: 2062\n",
      "         Function evaluations: 2823\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1863.104161\n",
      "         Iterations: 2479\n",
      "         Function evaluations: 3383\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1181.303636\n",
      "         Iterations: 2557\n",
      "         Function evaluations: 3456\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1079.044840\n",
      "         Iterations: 2263\n",
      "         Function evaluations: 3062\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1084.928019\n",
      "         Iterations: 1230\n",
      "         Function evaluations: 1762\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1090.420605\n",
      "         Iterations: 2866\n",
      "         Function evaluations: 3791\n",
      "using lambda=10000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49954.616860\n",
      "         Iterations: 1811\n",
      "         Function evaluations: 2584\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 12573.030391\n",
      "         Iterations: 1751\n",
      "         Function evaluations: 2428\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10778.384749\n",
      "         Iterations: 2776\n",
      "         Function evaluations: 3744\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10601.280505\n",
      "         Iterations: 2894\n",
      "         Function evaluations: 3871\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10531.790175\n",
      "         Iterations: 2158\n",
      "         Function evaluations: 2933\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10705.644771\n",
      "         Iterations: 2866\n",
      "         Function evaluations: 3817\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10803.364816\n",
      "         Iterations: 2685\n",
      "         Function evaluations: 3604\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10903.020042\n",
      "         Iterations: 1426\n",
      "         Function evaluations: 1989\n",
      "using lambda=100000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 132719.186621\n",
      "         Iterations: 1337\n",
      "         Function evaluations: 2016\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 104327.004395\n",
      "         Iterations: 1556\n",
      "         Function evaluations: 2256\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 103075.835177\n",
      "         Iterations: 2069\n",
      "         Function evaluations: 2847\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 104005.753094\n",
      "         Iterations: 2489\n",
      "         Function evaluations: 3371\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 105004.034590\n",
      "         Iterations: 1881\n",
      "         Function evaluations: 2578\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 106003.721268\n",
      "         Iterations: 1340\n",
      "         Function evaluations: 1911\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 107003.146319\n",
      "         Iterations: 2020\n",
      "         Function evaluations: 2753\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 108003.005775\n",
      "         Iterations: 1702\n",
      "         Function evaluations: 2373\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 109002.990377\n",
      "         Iterations: 918\n",
      "         Function evaluations: 1397\n",
      "using lambda=1000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1006433.656574\n",
      "         Iterations: 2232\n",
      "         Function evaluations: 3118\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1070008.346938\n",
      "         Iterations: 2595\n",
      "         Function evaluations: 3475\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1080001.598450\n",
      "         Iterations: 1311\n",
      "         Function evaluations: 1882\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1090001.164794\n",
      "         Iterations: 1589\n",
      "         Function evaluations: 2218\n",
      "using lambda=10000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10015774.578199\n",
      "         Iterations: 2184\n",
      "         Function evaluations: 3184\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10500096.883082\n",
      "         Iterations: 2696\n",
      "         Function evaluations: 3591\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10600016.812956\n",
      "         Iterations: 1426\n",
      "         Function evaluations: 2041\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10700008.399018\n",
      "         Iterations: 973\n",
      "         Function evaluations: 1468\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10800001.665477\n",
      "         Iterations: 2164\n",
      "         Function evaluations: 2965\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10900000.087330\n",
      "         Iterations: 857\n",
      "         Function evaluations: 1338\n",
      "using lambda=100000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 100009282.050760\n",
      "         Iterations: 1828\n",
      "         Function evaluations: 2672\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 103000002.705748\n",
      "         Iterations: 1939\n",
      "         Function evaluations: 2631\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 104000001.141175\n",
      "         Iterations: 2089\n",
      "         Function evaluations: 2808\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 105000001.084450\n",
      "         Iterations: 2026\n",
      "         Function evaluations: 2748\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 106000001.061780\n",
      "         Iterations: 1141\n",
      "         Function evaluations: 1648\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 107000001.057407\n",
      "         Iterations: 976\n",
      "         Function evaluations: 1459\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 108000001.054747\n",
      "         Iterations: 917\n",
      "         Function evaluations: 1383\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 109000001.053595\n",
      "         Iterations: 684\n",
      "         Function evaluations: 1091\n",
      "using lambda=1000000000.0\n",
      "Warning: Maximum number of function evaluations has been exceeded.\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy\n",
    "from scipy.spatial import distance\n",
    "\n",
    "num_f=X_train.shape[1]\n",
    "\n",
    "x = X_train.values[0].reshape(-1,num_f)\n",
    "y = rf.predict(x)\n",
    "t = 1-y\n",
    "eps = 0.01\n",
    "\n",
    "def d(a,b):\n",
    "    return distance.cityblock(a, b)\n",
    "\n",
    "def cf_loss(xp):\n",
    "    xp = xp.reshape(-1,num_f)\n",
    "    y_c = rf.predict(xp)\n",
    "    diff = abs(y_c-t)\n",
    "    distance = d(xp, x)\n",
    "    ft = l * (diff)**2\n",
    "    loss = ft + distance\n",
    "    return loss[0]\n",
    "\n",
    "b = []\n",
    "for c in range(num_f):\n",
    "    b.append((X_train.values[:,c].min(), X_train.values[:,c].max())) \n",
    "\n",
    "for m in numpy.arange(-4, 10):\n",
    "    l = 10.0**m\n",
    "    i = 0\n",
    "    print('using lambda='+str(l))\n",
    "    idx = random.randint(0, len(X_train.values)-1)\n",
    "    x_sample = X_train.values[idx].reshape(-1, num_f)\n",
    "    while abs(rf.predict(x_sample.reshape(-1,num_f)) - t) > 0.01 and i < 10:\n",
    "        if rf.predict(x_sample.reshape(-1,num_f)) > 0.5:\n",
    "          print('success for '+str(x_sample))\n",
    "        cf_min = optimize.fmin(cf_loss, [x_sample])\n",
    "        x_sample = cf_min\n",
    "        cur_loss = cf_loss(x_sample)\n",
    "        l += 10.0**(m-2)\n",
    "        i += 1\n",
    "\n",
    "    if rf.predict(x_sample.reshape(-1,num_f)) == t:\n",
    "        print(\"original input:\"+str(x))\n",
    "        print(\"counterf input:\"+str(x_sample))\n",
    "        print(\"counterf output:\"+str(rf.predict(x_sample.reshape(-1,num_f))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample cf instances directly from the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using lambda=1e-10\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 86790.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1e-09\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 16\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1e-08\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 85713.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 36\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 69138.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 36\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1e-07\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 16\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1e-06\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 15\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.000001\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1e-05\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 16\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000011\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000011\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000011\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.000011\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=0.0001\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 69138.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 36\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=0.001\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 42\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001010\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001020\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001030\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001040\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001050\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001060\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001070\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001080\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 39051.001090\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=0.01\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 85713.010000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 36\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 69138.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 36\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=0.1\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.100000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.101000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.102000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.103000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.104000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.105000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.106000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.107000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.108000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 44230.109000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.000000\n",
      "         Iterations: 16\n",
      "         Function evaluations: 45\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.010000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.020000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.030000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.040000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.050000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.060000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.070000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.080000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32655.090000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=10.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.100000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.200000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.300000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.400000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.500000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.600000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.700000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.800000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 32664.900000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 44\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=100.0\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1010.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1020.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1030.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1040.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1050.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1060.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1070.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1080.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1090.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=10000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49051.000000\n",
      "         Iterations: 15\n",
      "         Function evaluations: 42\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49151.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49251.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49351.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49451.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49551.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49651.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49751.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49851.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 49951.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=100000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 262563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 263563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 264563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 265563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 266563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 267563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 268563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 269563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 270563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 271563.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1039051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1049051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1059051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1069051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1079051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1089051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1099051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1109051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1119051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1129051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=10000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10039051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10139051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10239051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10339051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10439051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10539051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10639051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10739051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10839051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 10939051.000000\n",
      "         Iterations: 14\n",
      "         Function evaluations: 41\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=100000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 100000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 101000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 102000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 103000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 104000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 105000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 106000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 107000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 108000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 109000000.000000\n",
      "         Iterations: 3\n",
      "         Function evaluations: 8\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n",
      "using lambda=1000000000.0\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1000100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1010100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1020100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1030100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1040100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1050100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1060100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1070100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1080100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 1090100565.000000\n",
      "         Iterations: 13\n",
      "         Function evaluations: 38\n",
      "original input:[[1.65600e+03 1.00000e+01 6.14000e+03 3.24200e+03 3.80000e+01 2.00700e+03\n",
      "  2.00700e+03 2.00700e+03 1.00000e+00 2.00700e+03 2.00700e+03 1.00000e+00\n",
      "  2.00000e+00 3.00000e+00 7.33799e+05 5.00000e+04 0.00000e+00 4.51960e+04\n",
      "  5.00000e+04 4.25000e+04]]\n",
      "counterf input:[8.52613969e+03 7.85101448e+01 1.24419585e+04 1.21459917e+03\n",
      " 3.11321673e+01 3.48300551e+03 2.38722171e+03 3.33638772e+03\n",
      " 1.59525434e+00 2.03569111e+03 1.57256046e+03 3.42876887e+00\n",
      " 4.54851747e+00 4.62973366e+00 7.26269516e+05 5.14008091e+04\n",
      " 2.55153964e-03 4.16900597e+03 6.24120547e+04 4.25002672e+04]\n",
      "counterf output:[1]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "import numpy\n",
    "from scipy.spatial import distance\n",
    "\n",
    "num_f=X_train.shape[1]\n",
    "\n",
    "x = X_train.values[0].reshape(-1,num_f)\n",
    "y = rf.predict(x)\n",
    "t = 1-y\n",
    "eps = 0.01\n",
    "\n",
    "def d(a,b):\n",
    "    return distance.cityblock(a, b)\n",
    "\n",
    "def cf_loss(idx):\n",
    "    try:\n",
    "        idx = idx[0]\n",
    "    except:\n",
    "        pass\n",
    "    idx = int(idx)\n",
    "    xp = X_train.values[idx].reshape(-1, num_f)\n",
    "    y_c = rf.predict(xp)\n",
    "    diff = y_c-t\n",
    "    distance = d(xp, x)\n",
    "    loss = l * (diff)**2 + distance\n",
    "    return loss[0]\n",
    "\n",
    "b = [(0, X_train.shape[0])]\n",
    "\n",
    "for m in numpy.arange(-10, 10):\n",
    "    l = 10.0**m\n",
    "    i = 0\n",
    "    print('using lambda='+str(l))\n",
    "    idx_sample = random.randint(0, num_f-1)\n",
    "    while abs(t-rf.predict(X_train.values[idx_sample].reshape(-1, num_f))) > eps and i < 10:\n",
    "        cf_min = optimize.fmin(cf_loss, [idx_sample])\n",
    "        idx_sample = int(cf_min)\n",
    "        cur_loss = cf_loss(idx_sample)\n",
    "        l += 10.0**(m-2)\n",
    "        i += 1\n",
    "\n",
    "    if rf.predict(x_sample.reshape(-1,num_f)) == t:\n",
    "        print(\"original input:\"+str(x))\n",
    "        print(\"counterf input:\"+str(x_sample))\n",
    "        print(\"counterf output:\"+str(rf.predict(x_sample.reshape(-1,num_f))))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
